package com.lizb.pachong.main;

import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.lizb.pachong.link.Links;
import com.lizb.pachong.page.Page;
import com.lizb.pachong.page.RequestAndResponseTool;
import com.lizb.pachong.util.FileTool;

public class BaiduTuPianCrawler {

    /**
     * 使用种子初始化 URL 队列
     *
     * @param seeds 种子 URL
     * @return
     */
    private void initCrawlerWithSeeds(String[] seeds) {
        for (int i = 0; i < seeds.length; i++){
            Links.addUnvisitedUrlQueue(seeds[i]);
        }
    }


    /**
     * 抓取过程
     *
     * @param seeds
     * @return
     */
    public void crawling(String[] seeds) {

        //初始化 URL 队列
        initCrawlerWithSeeds(seeds);

        //循环条件：待抓取的链接不空且抓取的网页不多于 1000
        while (!Links.unVisitedUrlQueueIsEmpty()  && Links.getVisitedUrlNum() <= 1000) {

            //先从待访问的序列中取出第一个；
            String visitUrl = (String) Links.removeHeadOfUnVisitedUrlQueue();
            if (visitUrl == null){
                continue;
            }

            //根据URL得到page;
            Page page = RequestAndResponseTool.sendRequstAndGetResponse(visitUrl);
            String html = page.getHtml();
            if(html.indexOf("{") == 0){

                JSONObject json = JSONObject.parseObject(html);
                JSONArray data = json.getJSONArray("data");
                for (Object obj : data) {
                    JSONObject jsonObj = (JSONObject) obj;
                    String link = jsonObj.getString("thumbURL");
                    Links.addUnvisitedUrlQueue(link);
                    System.out.println("新增爬取路径: " + link);
                }
            } else {
                //将保存文件
                FileTool.saveToLocal(page);
            }

            //将已经访问过的链接放入已访问的链接中；
            Links.addVisitedUrlSet(visitUrl);
        }
    }


    public static void main(String[] args) {
        String[] strs = new String[10];
        for (int i = 0; i < 10; i++) {
            strs[i] = "https://image.baidu.com/search/acjson?tn=resultjson_com&ipn=rj&ct=201326592&is=&fp=result&queryWord=%E7%BE%8E%E5%A5%B3&cl=2&lm=-1&ie=utf-8&oe=utf-8&adpicid=&st=-1&z=&ic=&hd=&latest=&copyright=&word=%E7%BE%8E%E5%A5%B3&s=&se=&tab=&width=&height=&face=0&istype=2&qc=&nc=1&fr=&expermode=&force=&cg=girl" +
                    "&pn="+ (i * 30) +"&rn=30&gsm=5a&"+ (1560418012353L + i) +"=";
        }

        BaiduTuPianCrawler crawler = new BaiduTuPianCrawler();
        crawler.crawling(strs);
    }
}